#include "tokenizer.h"

bool isUTF8Letter(char c) {
  return (isalpha(c) || !(isascii(c)));
}

void Tokenizer::InitFromFile(string fn) {
  char * block;
  uint64 length;
  Mmap(fn, &block, &length);
  Init(block, block + length);
}

void Tokenizer::InitFromBlock(Memblock m) {
  Init(m.data_, m.data_ + m.size_);
}

// Tokenize XML that contains HTML mostly
void XMLTokenizer::NextToken() {
  /*
  // Recognize whitespace tokens.  Points into an arbitrary " " string.
  if (isspace(*current_)) {
    while (isspace(*current_)) current_++;
    token_.type_ = WHITESPACE_TOK;
    token_.start_ = (char *)" ";
    token_.end_ = token_.start_ + 1;
    return; }*/
  while (isspace(*current_)) current_++;
  if (*current_ == '\0') {
    token_.type_ = END_TOK;
    return;
  }
	 
  CHECK(*current_);
  token_.start_ = current_;
  
  // HTML escapes
  if (*current_ == '&') {
    do { current_++; } while (*current_ && (current_[-1] != ';'));
    token_.type_ = ESCAPE_TOK;
  } else 

  // Recognize XML tags
  if (*current_ == '<') {
    do { current_++; } while (*current_ && (current_[-1] != '>'));
    token_.type_ = TAG_TOK;
  } else 

  // Recognize numbers
  if (isdigit(*current_)) {
    while (isdigit(*current_)) current_++;
    token_.type_ = NUMBER_TOK;
  } else 

  // Recognize words
  if (isUTF8Letter(*current_)) {
    while (isUTF8Letter(*current_)) current_++;
    token_.type_ = WORD_TOK;
  } else {
  
  // Character or multicharacter tokens
  char c = *current_;
  token_.type_ = PUNCTUATION_TOK;
  while (*current_ == c) current_++;
  }
  token_.end_ = current_;
  while (isspace(*current_)) current_++;
}

// Reads the file "data" and tokenizes it
int tokenize_main() {
  XMLTokenizer xmlt;
  xmlt.InitFromFile("data");
  Token * tok = NULL;
  while ((tok = xmlt.GetNextToken())) {
    cout << "Token type: " << (int) tok->type_ << " ";
    if (tok->type_ != WHITESPACE_TOK) 
      cout << string((char*)tok->start_, (tok->end_ - tok->start_)) << endl;
    else cout << endl;
  }
  return 0;
}
